Library loading¶

In [1]:
import scanpy as sc, anndata as ad, numpy as np, pandas as pd
from scipy import sparse
from anndata import AnnData
import warnings
import socket
import plotly.express as px
from matplotlib import pylab
import sys
import yaml
import os
import matplotlib.pyplot as plt
import scvelo as scv
from pybiomart import Server
import rpy2
from pandas.api.types import CategoricalDtype
import rpy2.robjects as ro
import seaborn as sns
from matplotlib_venn import venn2, venn2_circles, venn2_unweighted
from matplotlib_venn import venn3, venn3_circles
from matplotlib import pyplot as plt
import plotly.graph_objects as go
import ipyparams


warnings.filterwarnings('ignore')
In [2]:
outdir="./outdir"
FinaLeaf = "/Cycling"
output_basename = "09C_Cycling_pBulk_bySegment.DEA"
mappingDict = {}
categoriesOrder = ["piece1","piece2","piece3","distal","medial","proximal"]

groupingCovariate = "group"
PseudooReplicates_per_group = 10

markers = "./data/resources/F_T.markers.scored.tsv"


totalPath = outdir+FinaLeaf+"/5C_Cycling_pBulk.bySegment."+str(PseudooReplicates_per_group)+"PRs.tsv"
adataPath = outdir+FinaLeaf+"/5C_Cycling_pBulk.bySegment."+str(PseudooReplicates_per_group)+"PRs.h5ad"

badhuriMarkers = ["PAX6","DDIT3","NEUROG2","CNBP","HMGB2","ZNF707","CAMTA1","SUB1","THYN1","NEUROG1","HMGB3","NFIX","BCL11A","TCF4","NR2F1"]
In [3]:
%load_ext rpy2.ipython
In [4]:
%%R
library(edgeR)
library(org.Hs.eg.db)
library(AnnotationDbi)
library(stats)
library(topGO)
R[write to console]: Loading required package: limma

R[write to console]: Loading required package: AnnotationDbi

R[write to console]: Loading required package: stats4

R[write to console]: Loading required package: BiocGenerics

R[write to console]: Loading required package: parallel

R[write to console]: 
Attaching package: ‘BiocGenerics’


R[write to console]: The following objects are masked from ‘package:parallel’:

    clusterApply, clusterApplyLB, clusterCall, clusterEvalQ,
    clusterExport, clusterMap, parApply, parCapply, parLapply,
    parLapplyLB, parRapply, parSapply, parSapplyLB


R[write to console]: The following object is masked from ‘package:limma’:

    plotMA


R[write to console]: The following objects are masked from ‘package:stats’:

    IQR, mad, sd, var, xtabs


R[write to console]: The following objects are masked from ‘package:base’:

    anyDuplicated, append, as.data.frame, basename, cbind, colnames,
    dirname, do.call, duplicated, eval, evalq, Filter, Find, get, grep,
    grepl, intersect, is.unsorted, lapply, Map, mapply, match, mget,
    order, paste, pmax, pmax.int, pmin, pmin.int, Position, rank,
    rbind, Reduce, rownames, sapply, setdiff, sort, table, tapply,
    union, unique, unsplit, which.max, which.min


R[write to console]: Loading required package: Biobase

R[write to console]: Welcome to Bioconductor

    Vignettes contain introductory material; view with
    'browseVignettes()'. To cite Bioconductor, see
    'citation("Biobase")', and for packages 'citation("pkgname")'.


R[write to console]: Loading required package: IRanges

R[write to console]: Loading required package: S4Vectors

R[write to console]: 
Attaching package: ‘S4Vectors’


R[write to console]: The following object is masked from ‘package:base’:

    expand.grid


R[write to console]: 

R[write to console]: Loading required package: graph

R[write to console]: Loading required package: GO.db

R[write to console]: 

R[write to console]: Loading required package: SparseM

R[write to console]: 
Attaching package: ‘SparseM’


R[write to console]: The following object is masked from ‘package:base’:

    backsolve


R[write to console]: 
groupGOTerms: 	GOBPTerm, GOMFTerm, GOCCTerm environments built.

R[write to console]: 
Attaching package: ‘topGO’


R[write to console]: The following object is masked from ‘package:IRanges’:

    members


In [5]:
%matplotlib inline
In [6]:
sc.settings.verbosity = 3         # verbosity: errors (0), warnings (1), info (2), hints (3)
sc.logging.print_header()
sc.settings.set_figure_params(dpi=50, facecolor='white')
pylab.rcParams['figure.figsize'] = (10, 10)
scanpy==1.8.1 anndata==0.7.6 umap==0.4.6 numpy==1.20.2 scipy==1.6.3 pandas==1.2.4 scikit-learn==0.24.2 statsmodels==0.13.1 python-igraph==0.9.8 louvain==0.7.1 pynndescent==0.5.5

Configure paths¶

In [7]:
hostRoot = "-".join(socket.gethostname().split('-')[0:2])



#indir=paths["paths"]["indir"][hostRoot]
#projectBaseDir=paths["paths"]["projectBaseDir"][hostRoot]

Load ExcM neurons pseudobulk and set categories order¶

In [8]:
adata = sc.read_h5ad(adataPath)
adata.obs[groupingCovariate] = adata.obs[groupingCovariate].astype(CategoricalDtype(categories=categoriesOrder, ordered=True))
In [9]:
total = pd.read_csv(totalPath, sep="\t", index_col=0)

Reorder counts cols¶

In [10]:
#adata = adata[adata.obs["group"].isin(RelevantAreas)]
total = total[adata.obs_names]

DEA¶

RPY INITIALIZATION¶

In [11]:
edgeR_topTags = ro.r['topTags']
edgeR_glmLRT = ro.r['glmLRT']
edgeR_glmQLFTest = ro.r['glmQLFTest']
as_data_frame = ro.r['as.data.frame']
rprint = rpy2.robjects.globalenv.find("print")
In [12]:
DEGs = {}
TestCov = groupingCovariate
In [13]:
adata.obs.group.value_counts()
Out[13]:
piece1      10
piece2      10
piece3      10
distal      10
medial      10
proximal    10
Name: group, dtype: int64

MM¶

In [14]:
obs = adata.obs[[TestCov,"pseudoreplicate"]].copy()
obs["pseudoreplicate"] = obs.index.tolist()
obs["TestCov"] = obs[TestCov].astype(str)

#obs["TestCov"] = obs[TestCov]

Counts¶

In [15]:
totalRelevant = total[obs.index]
totalRelevant.head()
Out[15]:
distal_0 distal_1 distal_2 distal_3 distal_4 distal_5 distal_6 distal_7 distal_8 distal_9 ... proximal_0 proximal_1 proximal_2 proximal_3 proximal_4 proximal_5 proximal_6 proximal_7 proximal_8 proximal_9
MIR1302-2HG 0.000000 0.0 0.000000 0.0 0.000000 0.0 0.0 0.0 0.000000 0.0 ... 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
FAM138A 0.000000 0.0 0.000000 0.0 0.000000 0.0 0.0 0.0 0.000000 0.0 ... 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
OR4F5 0.000000 0.0 0.000000 0.0 0.000000 0.0 0.0 0.0 0.000000 0.0 ... 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
AL627309.1 1.909059 0.0 1.303118 0.0 1.871744 0.0 0.0 0.0 0.475036 0.0 ... 0.0 0.724288 3.736357 1.517416 0.611369 0.897955 0.747047 0.500132 2.488713 1.779022
AL627309.3 0.000000 0.0 0.000000 0.0 0.000000 0.0 0.0 0.0 0.000000 0.0 ... 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000

5 rows × 60 columns

In [16]:
minCounts = totalRelevant[totalRelevant > 0].min(axis = 1).min()

#Universe 1 count in at least 5% of samples per group
groupThreshold = round(pd.get_dummies(obs[TestCov]).T.sum(axis = 1) *0.05)
bolMatrix = (np.matrix(np.dot(pd.get_dummies(obs[TestCov]).T, (totalRelevant > 0).astype(int).T)) - np.matrix(groupThreshold).T > 0)
bolVect = (bolMatrix.sum(axis = 0) >= 1).A1
totalRelevant = totalRelevant.loc[bolVect]
len(totalRelevant)
Out[16]:
25569
In [17]:
universe = totalRelevant.index.tolist()
In [18]:
levelsToMap = adata.obs[groupingCovariate].cat.categories.tolist()
levelsToMap
Out[18]:
['piece1', 'piece2', 'piece3', 'distal', 'medial', 'proximal']
In [19]:
%%R -i obs -i totalRelevant -i levelsToMap -o dds 

mmVector<- factor(obs$TestCov, levels = c(levelsToMap))

mm <- model.matrix(~mmVector )
row.names(mm) <- colnames(totalRelevant)

dds <- DGEList(totalRelevant, group = obs$TestCov, genes = rownames(totalRelevant))

#Ennotate with entrez genes
anno <- select(org.Hs.eg.db, keys=rownames(dds$counts),columns=c("ENTREZID","SYMBOL"),keytype="SYMBOL")
colnames(anno) <- c("genes","entrez")
anno <- anno[!duplicated(anno$genes, fromLast=T), ]
dds$genes$entrez <- merge(x = data.frame(dds$genes), y = anno, by = "genes", all.x = TRUE)$entrez

dim(dds)
R[write to console]: 'select()' returned 1:many mapping between keys and columns

[1] 25569    60
In [20]:
%%R

mm
           (Intercept) mmVectorpiece2 mmVectorpiece3 mmVectordistal
distal_0             1              0              0              1
distal_1             1              0              0              1
distal_2             1              0              0              1
distal_3             1              0              0              1
distal_4             1              0              0              1
distal_5             1              0              0              1
distal_6             1              0              0              1
distal_7             1              0              0              1
distal_8             1              0              0              1
distal_9             1              0              0              1
medial_0             1              0              0              0
medial_1             1              0              0              0
medial_2             1              0              0              0
medial_3             1              0              0              0
medial_4             1              0              0              0
medial_5             1              0              0              0
medial_6             1              0              0              0
medial_7             1              0              0              0
medial_8             1              0              0              0
medial_9             1              0              0              0
piece1_0             1              0              0              0
piece1_1             1              0              0              0
piece1_2             1              0              0              0
piece1_3             1              0              0              0
piece1_4             1              0              0              0
piece1_5             1              0              0              0
piece1_6             1              0              0              0
piece1_7             1              0              0              0
piece1_8             1              0              0              0
piece1_9             1              0              0              0
piece2_0             1              1              0              0
piece2_1             1              1              0              0
piece2_2             1              1              0              0
piece2_3             1              1              0              0
piece2_4             1              1              0              0
piece2_5             1              1              0              0
piece2_6             1              1              0              0
piece2_7             1              1              0              0
piece2_8             1              1              0              0
piece2_9             1              1              0              0
piece3_0             1              0              1              0
piece3_1             1              0              1              0
piece3_2             1              0              1              0
piece3_3             1              0              1              0
piece3_4             1              0              1              0
piece3_5             1              0              1              0
piece3_6             1              0              1              0
piece3_7             1              0              1              0
piece3_8             1              0              1              0
piece3_9             1              0              1              0
proximal_0           1              0              0              0
proximal_1           1              0              0              0
proximal_2           1              0              0              0
proximal_3           1              0              0              0
proximal_4           1              0              0              0
proximal_5           1              0              0              0
proximal_6           1              0              0              0
proximal_7           1              0              0              0
proximal_8           1              0              0              0
proximal_9           1              0              0              0
           mmVectormedial mmVectorproximal
distal_0                0                0
distal_1                0                0
distal_2                0                0
distal_3                0                0
distal_4                0                0
distal_5                0                0
distal_6                0                0
distal_7                0                0
distal_8                0                0
distal_9                0                0
medial_0                1                0
medial_1                1                0
medial_2                1                0
medial_3                1                0
medial_4                1                0
medial_5                1                0
medial_6                1                0
medial_7                1                0
medial_8                1                0
medial_9                1                0
piece1_0                0                0
piece1_1                0                0
piece1_2                0                0
piece1_3                0                0
piece1_4                0                0
piece1_5                0                0
piece1_6                0                0
piece1_7                0                0
piece1_8                0                0
piece1_9                0                0
piece2_0                0                0
piece2_1                0                0
piece2_2                0                0
piece2_3                0                0
piece2_4                0                0
piece2_5                0                0
piece2_6                0                0
piece2_7                0                0
piece2_8                0                0
piece2_9                0                0
piece3_0                0                0
piece3_1                0                0
piece3_2                0                0
piece3_3                0                0
piece3_4                0                0
piece3_5                0                0
piece3_6                0                0
piece3_7                0                0
piece3_8                0                0
piece3_9                0                0
proximal_0              0                1
proximal_1              0                1
proximal_2              0                1
proximal_3              0                1
proximal_4              0                1
proximal_5              0                1
proximal_6              0                1
proximal_7              0                1
proximal_8              0                1
proximal_9              0                1
attr(,"assign")
[1] 0 1 1 1 1 1
attr(,"contrasts")
attr(,"contrasts")$mmVector
[1] "contr.treatment"

In [21]:
%%R -i minCounts

#keep <- filterByExpr(dds, min.count = minCounts, min.total.count = minCounts*5)
#dds <- dds[keep,,keep.lib.sizes=FALSE]
dim(dds)
[1] 25569    60
In [22]:
%%R
dds <- calcNormFactors(dds)
dds <- estimateGLMRobustDisp(dds, mm)
fit <- glmFit(dds, mm)

MAP contrasts!¶

In [23]:
levelsToMap
Out[23]:
['piece1', 'piece2', 'piece3', 'distal', 'medial', 'proximal']
In [24]:
%%R -o piece2_vs_piece1 -o piece3_vs_piece1  -o distal_vs_piece1  -o medial_vs_piece1  -o proximal_vs_piece1 -o proximal_vs_distal -o medial_vs_distal
#-o Mid_vs_early -o Late_vs_mid
topn=10000

piece2_vs_piece1 = topTags(glmLRT(fit, coef=2), adjust.method = "bonferroni",  n = topn, sort.by = "logFC")
piece3_vs_piece1 = topTags(glmLRT(fit, coef=3), adjust.method = "bonferroni", n = topn, sort.by = "logFC")
distal_vs_piece1 = topTags(glmLRT(fit, coef=4), adjust.method = "bonferroni", n = topn, sort.by = "logFC")
medial_vs_piece1 = topTags(glmLRT(fit, coef=5), adjust.method = "bonferroni",  n = topn, sort.by = "logFC")
proximal_vs_piece1 = topTags(glmLRT(fit, coef=6), adjust.method = "bonferroni",  n = topn, sort.by = "logFC")
proximal_vs_distal = topTags(glmLRT(fit, contrast=c(0,0,0,-1,0,1)), adjust.method = "bonferroni",  n = topn, sort.by = "logFC")
medial_vs_distal = topTags(glmLRT(fit, contrast=c(0,0,0,-1,1,0)), adjust.method = "bonferroni",  n = topn, sort.by = "logFC")
In [25]:
DEGsDict = {"piece2_vs_piece1":piece2_vs_piece1,
            "piece3_vs_piece1":piece3_vs_piece1,
            "distal_vs_piece1":distal_vs_piece1,
            "medial_vs_piece1":medial_vs_piece1,
            "proximal_vs_piece1":proximal_vs_piece1,
            "proximal_vs_distal":proximal_vs_distal,
            "medial_vs_distal":medial_vs_distal,
           }

Volcanos.¶

In [26]:
markersDF = pd.read_csv(markers, header=None, sep = "\t", names=["name","area","score"])
temporalMarkers = markersDF.loc[markersDF["area"] == "Temporal","name"]
frontalMarkers = markersDF.loc[markersDF["area"] == "Frontal","name"]
In [27]:
LOGCFTHRESHOLD = 1.5
for k in list(DEGsDict.keys()):
    

    DEGsDict[k] = rpy2.robjects.pandas2ri.rpy2py(as_data_frame(DEGsDict[k]))
    DEGsDict[k]["-logPVal"] = -np.log10(DEGsDict[k].PValue)
    DEGsDict[k]["significant"] = "notSignificant"
    DEGsDict[k].loc[(DEGsDict[k]["logFC"] < -LOGCFTHRESHOLD) &  (DEGsDict[k]["FWER"] < 0.01),"significant"] = "Down"
    DEGsDict[k].loc[(DEGsDict[k]["logFC"] > LOGCFTHRESHOLD) &  (DEGsDict[k]["FWER"] < 0.01),"significant"] = "Up"
    #DEGsDict[k]["knownMarker"] = "unlisted"
    #DEGsDict[k].loc[set(frontalMarkers).intersection(DEGsDict[k].index),"knownMarker"] = "frontal"
    #DEGsDict[k].loc[set(temporalMarkers).intersection(DEGsDict[k].index),"knownMarker"] = "temporal"



    color_discrete_map = {'notSignificant': 'white', 'Up': 'red', 'Down': 'blue'}

    fig = px.scatter(DEGsDict[k], x="logFC", y="-logPVal",
                     #hover_data=DEGsDict[k][["genes","knownMarker"]], 
                     hover_data=DEGsDict[k][["genes"]], 
                     width=600, height=600, 
                     color = "significant", color_discrete_map=color_discrete_map,
                     #symbol="knownMarker",
                     title=k,
                    template="simple_white")
    
    
    fig.update_traces(mode='markers', marker_line_width=1, marker_size=10)
    
    fig.add_hline(y=DEGsDict[k].loc[DEGsDict[k]["FWER"] - 0.01 < 0,"-logPVal" ].min(), 
                  line_color="black",line_width=2, line_dash="dash")

    fig.add_vline(x=LOGCFTHRESHOLD, 
                  line_color="black",line_width=2, line_dash="dash")
        
    fig.add_vline(x=-LOGCFTHRESHOLD, 
                  line_color="black",line_width=2, line_dash="dash")
    fig.show()

GO Contrast¶

Pick the contrast for GO and name to save as¶

In [28]:
contrast = "proximal_vs_distal"
GOname = "GO_proximal_vs_distal.tsv"
GOlist = DEGsDict[contrast][DEGsDict[contrast]["significant"] != "notSignificant"].index.tolist()
In [29]:
%%R -i GOlist -i universe -o results


Universe <- universe
Temporal_vs_PFCVector <- factor(as.integer(Universe%in%GOlist))
names(Temporal_vs_PFCVector) <- Universe

BPann <- inverseList(topGO::annFUN.org(whichOnto="BP", feasibleGenes=names(Temporal_vs_PFCVector),  
                                       mapping="org.Hs.eg.db", ID="symbol"))

    
GOdata <- new('topGOdata', ontology="BP", allGenes=Temporal_vs_PFCVector, annotat=annFUN.gene2GO, 
                         gene2GO=BPann, nodeSize=15)

GOtest <- runTest(GOdata, algorithm='weight01', statistic='fisher')


results <- topGO::GenTable(GOdata, Statistics=GOtest, topNodes=length(GOtest@score), numChar=100)
R[write to console]: 
Building most specific GOs .....

R[write to console]: 	( 11946 GO terms found. )

R[write to console]: 
Build GO DAG topology ..........

R[write to console]: 	( 15768 GO terms and 36719 relations. )

R[write to console]: 
Annotating nodes ...............

R[write to console]: 	( 14851 genes annotated to the GO terms. )

R[write to console]: 
			 -- Weight01 Algorithm -- 

		 the algorithm is scoring 3508 nontrivial nodes
		 parameters: 
			 test statistic: fisher

R[write to console]: 
	 Level 17:	4 nodes to be scored	(0 eliminated genes)

R[write to console]: 
	 Level 16:	8 nodes to be scored	(0 eliminated genes)

R[write to console]: 
	 Level 15:	19 nodes to be scored	(93 eliminated genes)

R[write to console]: 
	 Level 14:	34 nodes to be scored	(210 eliminated genes)

R[write to console]: 
	 Level 13:	68 nodes to be scored	(561 eliminated genes)

R[write to console]: 
	 Level 12:	123 nodes to be scored	(1398 eliminated genes)

R[write to console]: 
	 Level 11:	206 nodes to be scored	(3520 eliminated genes)

R[write to console]: 
	 Level 10:	345 nodes to be scored	(5218 eliminated genes)

R[write to console]: 
	 Level 9:	460 nodes to be scored	(7190 eliminated genes)

R[write to console]: 
	 Level 8:	531 nodes to be scored	(9131 eliminated genes)

R[write to console]: 
	 Level 7:	581 nodes to be scored	(11217 eliminated genes)

R[write to console]: 
	 Level 6:	505 nodes to be scored	(12838 eliminated genes)

R[write to console]: 
	 Level 5:	331 nodes to be scored	(13663 eliminated genes)

R[write to console]: 
	 Level 4:	186 nodes to be scored	(14242 eliminated genes)

R[write to console]: 
	 Level 3:	86 nodes to be scored	(14487 eliminated genes)

R[write to console]: 
	 Level 2:	20 nodes to be scored	(14592 eliminated genes)

R[write to console]: 
	 Level 1:	1 nodes to be scored	(14680 eliminated genes)

In [30]:
FilteredResults = results[results["Statistics"].astype(float) < 0.01]
FilteredResults.to_csv(outdir+FinaLeaf+"/"+output_basename+"."+GOname, sep = "\t")
FilteredResults.Term.tolist()[:20]
Out[30]:
['dentate gyrus development',
 'inner ear morphogenesis',
 'positive regulation of transcription by RNA polymerase II',
 'neuron fate commitment',
 'negative regulation of transcription by RNA polymerase II',
 'otic vesicle development',
 'forebrain neuron development',
 'anterior/posterior pattern specification',
 'extracellular matrix organization',
 'negative chemotaxis',
 'positive regulation of neuron differentiation',
 'neuron development',
 'cerebral cortex development',
 'neuron migration',
 'forebrain regionalization',
 'retina morphogenesis in camera-type eye',
 'axon guidance',
 'cell fate determination',
 'heterophilic cell-cell adhesion via plasma membrane cell adhesion molecules',
 'epithelial cell fate commitment']

Double check for results¶

In [31]:
UP = DEGsDict[contrast][DEGsDict[contrast]["significant"] != "notSignificant"].sort_values("logFC", ascending = False)
UP = UP[UP.logFC > 0].index.tolist()
In [32]:
DOWN = DEGsDict[contrast][DEGsDict[contrast]["significant"] != "notSignificant"].sort_values("logFC", ascending = True)
DOWN = DOWN[DOWN.logFC < 0].index.tolist()
In [33]:
print("Upregulated in contrast "+contrast)

sc.pl.violin(adata, 
             keys = UP[:4] , 
             groupby=groupingCovariate)
Upregulated in contrast proximal_vs_distal
In [34]:
print("Downregolated in contrast "+contrast)
sc.pl.violin(adata, 
             keys = DOWN[:4] , 
             groupby=groupingCovariate)
Downregolated in contrast proximal_vs_distal

Save¶

In [35]:
for k in DEGsDict.keys():
    DEGsDict[k][DEGsDict[k].significant != "notSignificant" ].sort_values("logFC", ascending = False).to_csv(outdir+FinaLeaf+"/"+output_basename+"."+ k + ".tsv", sep = "\t")
    DEGsDict[k][DEGsDict[k].significant != "notSignificant" ].sort_values("logFC", ascending = False).to_excel(outdir+FinaLeaf+"/"+output_basename+"."+ k + ".xls")